# -*- coding: utf-8 -*-
"""
Created on Sat Sep 17 10:47:26 2022

@author: 123
"""

#异常值检测
new_make_gaussian_kmeans2_outlier_df=(new_make_gaussian_kmeans2_df.iloc[:,:-2]).iloc[:,1:]
new_make_gaussian_kmeans1_outlier_df=(new_make_gaussian_kmeans1_df.iloc[:,:-2]).iloc[:,1:]
new_make_gaussian_kmeans0_outlier_df=(new_make_gaussian_kmeans0_df.iloc[:,:-2]).iloc[:,1:]

# 重置index
new_make_gaussian_kmeans2_outlier_df=new_make_gaussian_kmeans2_outlier_df.reset_index(drop=True)
new_make_gaussian_kmeans1_outlier_df=new_make_gaussian_kmeans1_outlier_df.reset_index(drop=True)
new_make_gaussian_kmeans0_outlier_df=new_make_gaussian_kmeans0_outlier_df.reset_index(drop=True)

#将df转化为np

new_make_gaussian_kmeans2_outlier_np=np.array(new_make_gaussian_kmeans2_outlier_df)
new_make_gaussian_kmeans1_outlier_np=np.array(new_make_gaussian_kmeans1_outlier_df)
new_make_gaussian_kmeans0_outlier_np=np.array(new_make_gaussian_kmeans0_outlier_df)

# 使用训练集 训练异常值检测模型 
from sklearn.covariance import EmpiricalCovariance, MinCovDet
from sklearn.covariance import EllipticEnvelope


detector_kmeans2 = EllipticEnvelope() # 构造异常值识别器
detector_kmeans2.fit(new_make_gaussian_kmeans2_outlier_np) # 拟合识别器
detector_kmeans2_pre=detector_kmeans2.predict(new_make_gaussian_kmeans2_outlier_np) # 预测异常值


detector_kmeans1 = EllipticEnvelope() # 构造异常值识别器
detector_kmeans1.fit(new_make_gaussian_kmeans1_outlier_np) # 拟合识别器
detector_kmeans1_pre=detector_kmeans1.predict(new_make_gaussian_kmeans1_outlier_np) # 预测异常值

detector_kmeans0 = EllipticEnvelope() # 构造异常值识别器
detector_kmeans0.fit(new_make_gaussian_kmeans0_outlier_np) # 拟合识别器
detector_kmeans0_pre=detector_kmeans0.predict(new_make_gaussian_kmeans0_outlier_np) # 预测异常值


detector_kmeans2_pre_df=pd.DataFrame(detector_kmeans2_pre)
detector_kmeans1_pre_df=pd.DataFrame(detector_kmeans1_pre)
detector_kmeans0_pre_df=pd.DataFrame(detector_kmeans0_pre)


p_col=['is_outlier']
#y_pre_kmeans_df.columns=p_col

detector_kmeans2_pre_df.columns=p_col
detector_kmeans1_pre_df.columns=p_col
detector_kmeans0_pre_df.columns=p_col

#将数据 is_outlier值挂上
new_make_gaussian_kmeans2_outlier_df=pd.concat([new_make_gaussian_kmeans2_outlier_df,detector_kmeans2_pre_df],join="inner", axis=1)
new_make_gaussian_kmeans1_outlier_df=pd.concat([new_make_gaussian_kmeans1_outlier_df,detector_kmeans1_pre_df],join="inner", axis=1)
new_make_gaussian_kmeans0_outlier_df=pd.concat([new_make_gaussian_kmeans0_outlier_df,detector_kmeans0_pre_df],join="inner", axis=1)


new_make_gaussian_kmeans2_outlier_1_df=new_make_gaussian_kmeans2_outlier_df.query('is_outlier==1')

# 100387 
new_make_gaussian_kmeans1_outlier_1_df=new_make_gaussian_kmeans1_outlier_df.query('is_outlier==1')
# 69625
new_make_gaussian_kmeans0_outlier_1_df=new_make_gaussian_kmeans0_outlier_df.query('is_outlier==1')

# 69733 



# 69826

# kmeans	原始记录	最终需要数据	 高斯剔除异常值记录 高斯增加需要记录      
# 0	8568	60120	51552	     69733                  103104
# 1	8616	60457	51841	     69625                  103682
# 2	10960	76904	65944	     100387                 131888

0.7392 0.7445 0.6568

from sklearn.model_selection import train_test_split
new_make_gaussian_kmeans2_outlier_1_df_train, new_make_gaussian_kmeans2_outlier_1_df_test = train_test_split(new_make_gaussian_kmeans2_outlier_1_df, test_size=0.66, random_state=42)
new_make_gaussian_kmeans1_outlier_1_df_train, new_make_gaussian_kmeans1_outlier_1_df_test = train_test_split(new_make_gaussian_kmeans1_outlier_1_df, test_size=0.74, random_state=42)
new_make_gaussian_kmeans0_outlier_1_df_train, new_make_gaussian_kmeans0_outlier_1_df_test = train_test_split(new_make_gaussian_kmeans0_outlier_1_df, test_size=0.74, random_state=42)

# 合并数据

new_make_gaussian_outlier_1_df_test=pd.concat([new_make_gaussian_kmeans2_outlier_1_df_test,new_make_gaussian_kmeans1_outlier_1_df_test,new_make_gaussian_kmeans0_outlier_1_df_test])


#col_name=df1.columns.tolist()                   # 将数据框的列名全部提取出来存放在列表里
#col_name.insert(2,'city')                      # 在列索引为2的位置插入一列,列名为:city，刚插入时不会有值，整列都是NaN
#df1=df1.reindex(columns=col_name)              # DataFrame.reindex() 对原行/列索引重新构建索引值
new_make_gaussian_outlier_1_df_test_col=new_make_gaussian_outlier_1_df_test.columns.tolist()
new_make_gaussian_outlier_1_df_test_col.insert(0,'Risk_Flag')
new_make_gaussian_outlier_1_df_test=new_make_gaussian_outlier_1_df_test.reindex(columns=new_make_gaussian_outlier_1_df_test_col)
new_make_gaussian_outlier_1_df_test['Risk_Flag']=1


#最后数据
fin_data_df=pd.concat([new_df_not_outlier,new_make_gaussian_outlier_1_df_test])


fin_data_index_df=fin_data_df.reset_index(drop=True)





